#!/bin/bash

#Submit this script with: sbatch thefilename

#SBATCH --time=720:00:00   # walltime
#SBATCH --ntasks-per-node=64
#SBATCH --nodes=1   # only 1, do not change. 
#SBATCH --mem=500G   # memory per node
#SBATCH -J "vasp544"   # job name
#SBATCH --partition=vasp # do not change
#SBATCH --qos=vasp # do not change

## Set your vasp version here, default vasp_std
## options are: std, ncl, gam
export vaspVersion=std


### Do not change anything bellow ###

# LOAD MODULES, INSERT CODE, AND RUN YOUR PROGRAMS HERE
ulimit -d unlimited
ulimit -s unlimited
ulimit -t unlimited
ulimit -v unlimited

export SLURM_EXPORT_ENV=ALL


cd $SLURM_SUBMIT_DIR
username=`whoami`

### here, suppose /tmp is the I/O storage. 

VASP_RUNDIR=/tmp/${username}.${SLURM_JOB_ID}

if [ ! -a $VASP_RUNDIR ]; then
   echo "Scratch directory $VASP_RUNDIR created."
   mkdir -p $VASP_RUNDIR
fi

cp -r $SLURM_SUBMIT_DIR/* $VASP_RUNDIR

cd $VASP_RUNDIR


echo "Starting VASP run at `hostname` on:" `date` >> ${SLURM_JOB_ID}.log
export EXE="singularity exec /opt/hpc4you/apps/vasp/v544.vtst.solvation/vasp.v544.EXE vasp_${vaspVersion}"
srun --mpi=pmi2 ${EXE} 1>vasp.out 2>vasp.err

echo "Finished VASP run at `hostname` on:" `date` >> ${SLURM_JOB_ID}.log
cat $SLURM_JOB_NODELIST >> ${SLURM_JOB_ID}.log

mv $VASP_RUNDIR/* $SLURM_SUBMIT_DIR
rm -fr $VASP_RUNDIR

